package com.chance.cc.crawler.development.scripts.du;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.CategoryTag;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.BASE64Decoder;
import sun.misc.BASE64Encoder;

import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;

import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author lt
 * @version 1.0
 * @date 2021-04-28 10:25:34
 * @email okprog@sina.com
 */
public class DUSellCalenderCrawlerScript extends CrawlerCommonScript {

    private static Logger logger = LoggerFactory.getLogger(DUSellCalenderCrawlerScript.class);

    //开始链接
    public static final String startUrlRegex = "https://m\\.dewu\\.com/sns/v1/sell/index\\S*";
    //按月份搜索的链接
    public static final String searchMonthUrlRegex = "https://m\\.dewu\\.com/sns/v1/sell/list\\S*";
    public static final String searchMonthUrlFormat = "https://m.dewu.com/sns/v1/sell/list?sellTime=%s&startDate=&endDate=&categoryId=&lastId=&v=4.2.0&newSign=%s";
    //详情链接
    public static final String productDetailUrlRegex = "https://app\\.dewu\\.com/api/v1/\\S*/index/\\S*/flow/product/detail\\S*";
//    public static final String productDetailUrl = "https://app.dewu.com/api/v1/app/index/ice/flow/product/detail";//app
    //发售链接
    public static final String sellDetailUrlRegex = "https://m\\.dewu\\.com/mapi/sell/detail2\\S*";
    public static final String sellDetailUrlFormat = "https://m.dewu.com/mapi/sell/detail2?sellId=%s&sellPlace=&sign=%s";
    //商品明细url
    public static final String productDetailUrl = "https://app.dewu.com/api/v1/h5/index/fire/flow/product/detail";//h5
    public static final String sellProductUrl = "https://app.dewu.com/api/v1/h5/inventory/price/h5/queryBuyNowInfo";
    public static final String askProductUrl = "https://app.dewu.com/api/v1/h5/newbidding/buyer/view";
    //AES_ECB加密的密钥
    public static final String scKey = "d245a0ba8d678a61";
    public static final String NEW_SIGN = "newSign";
    public static final String SELL_DATE = "sell_date";
    public static final String SELL_ID = "sell_id";
    public static final String PRODUCT_ID = "product_id";
    public static final String DOMAIN = "du_app";
    public static final String SITE = "sell_calender";
    public static final String SITE_BIZ = "sell_calender";

    public static final String tagProductDetail = "product_detail";
    public static final String tagSellProduct = "sell_product";
    public static final String tagAskProduct = "ask_product";
    public static final String tagProductId = "product_id";


    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        HttpRequest httpRequest = requestRecord.getHttpRequest();
        String httpRequestUrl = httpRequest.getUrl();
        if (httpRequestUrl.matches(startUrlRegex) && !httpRequestUrl.contains(NEW_SIGN)){
            Map<String, Object> urlParams = getUrlParams(httpRequestUrl);
            if (null != urlParams){
                long timeMillis = System.currentTimeMillis();
                Map<String, Object> params = genSignNeedParams(urlParams, String.valueOf(timeMillis));
                String newSign = genNewSignature(params, scKey);
                urlParams.put(NEW_SIGN,newSign);
                httpRequest.setUrl(String.format("%s?%s",httpRequestUrl.split("\\?")[0],asUrlParams(urlParams)));
                httpRequest.setHeaders(genRequestHeaders(String.valueOf(timeMillis)));
            }
        }
        return super.prepareRequest(requestRecord, supportSourceRecords);
    }


    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();
        HttpRequest lastRequest = crawlerRequestRecord.getHttpRequest();
        if (404 == httpPage.getStatusCode()){
            logger.info("status code is 404,detail is [{}]",JSONObject.toJSONString(crawlerRequestRecord));
            crawlerRequestRecord.setNeedWashPage(false);
            return parsedLinks;
        }
        if (doHttpPageCheck(crawlerRequestRecord,httpPage)){
            if(httpPage.getStatusCode() == 403){
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage());
                }
            }

            crawlerRequestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
            addCrawlerRecord(crawlerRequestRecord,parsedLinks);
            crawlerRequestRecord.setNeedWashPage(false);
            return parsedLinks;
        }
        String lastRequestUrl = lastRequest.getUrl();
        if (lastRequestUrl.matches(startUrlRegex) && lastRequestUrl.contains(NEW_SIGN)){
            return parseIndexLinks(crawlerRequestRecord,httpPage,parsedLinks);
        }
        if (lastRequestUrl.matches(searchMonthUrlRegex)){
            return parseMonthLinks(crawlerRequestRecord,httpPage,parsedLinks);
        }
        if (lastRequestUrl.matches(productDetailUrlRegex)){
            return parseProductLinks(crawlerRequestRecord,httpPage,parsedLinks);
        }
        if (lastRequestUrl.matches(sellProductUrl)){
            return parseSellProductLinks(crawlerRequestRecord,httpPage,parsedLinks);
        }

        return parsedLinks;
    }

    private void addCrawlerRecord(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> parsedLinks) {
        CrawlerRequestRecord crawlerRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpRequest(crawlerRequestRecord.getHttpRequest())
                .recordKey(crawlerRequestRecord.getRecordKey())
                .httpHeads(crawlerRequestRecord.getHttpRequest().getHeaders())
                .releaseTime(System.currentTimeMillis())
                .needWashed(crawlerRequestRecord.isNeedWashPage())
                .needParsed(crawlerRequestRecord.isNeedParsedPage())
                .notFilterRecord()
                .copyBizTags()
                .build();
        parsedLinks.add(crawlerRecord);
    }

    private List<CrawlerRequestRecord> parseSellProductLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        try {
            JSONObject pageObj = JSONObject.parseObject(httpPage.getRawText());
            int code = pageObj.getIntValue("code");
            if (200 != code){
                String errorInfo = "du sell product detail download error,request url "+httpPage.getRequest().getUrl()
                        +",page rawtext "+httpPage.getRawText();
                logger.error(errorInfo);
                throw new Exception(errorInfo);
            }
            String spuId = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagProductId);
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagSellProduct,httpPage.getRawText());

            CrawlerRequestRecord crawlerItemRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .httpUrl(askProductUrl)
                    .httpHead("appid","wxapp")
                    .httpHead("content-type","application/json")
                    .recordKey(askProductUrl+spuId)
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(true)
//                    .needParsed(false)
                    .notFilterRecord()
                    .copyBizTags()
                    .build();

            HttpRequest httpRequest = crawlerItemRequestRecord.getHttpRequest();
            httpRequest.setMethod("POST");
            Map<String,Object> params = new HashMap<>();
            params.put("spuId",spuId);

            String sign = genSignature(params);
            params.put("sign",sign);
            HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
            httpRequest.setRequestBody(httpRequestBody);

            httpRequest.addHeader("Host","app.dewu.com");
            httpRequest.addHeader("Connection","keep-alive");
            httpRequest.addHeader("User-Agent",genRandomUA());
            httpRequest.addHeader("Content-Type","application/json");
            httpRequest.addHeader("platform","h5");
            httpRequest.addHeader("AppId","wxapp");
            httpRequest.addHeader("appVersion","4.4.0");
            httpRequest.addHeader("charset","utf-8");
            httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
            httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
            httpRequest.addHeader("wxapp-login-token","");

            parsedLinks.add(crawlerItemRequestRecord);
        }catch (Exception e){
            logger.error(e.getMessage());
            crawlerRequestRecord.setNeedWashPage(false);
            parsedLinks.add(crawlerRequestRecord);
        }

        return parsedLinks;
    }

    private List<CrawlerRequestRecord> parseProductLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        try {
            JSONObject pageObj = JSONObject.parseObject(httpPage.getRawText());
            int code = pageObj.getIntValue("code");
            if (200 != code){
                if (20801001 == code){
                    logger.info("商品不存在,detail is [{}]",JSONObject.toJSONString(crawlerRequestRecord));
                    return parsedLinks;
                }
                String errorInfo = "du product detail download error,request url "+httpPage.getRequest().getUrl() +",page rawtext "+httpPage.getRawText();
                logger.error(errorInfo);
                throw new Exception(errorInfo);
            }
            String spuId = pageObj.getJSONObject("data").getJSONObject("detail").getString("spuId");
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagProductDetail,httpPage.getRawText());
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(tagProductId,spuId);

            CrawlerRequestRecord crawlerItemRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .httpUrl(sellProductUrl)
                    .httpHead("appid","wxapp")
                    .httpHead("content-type","application/json")
                    .recordKey(sellProductUrl+spuId)
                    .releaseTime(System.currentTimeMillis())
                    .needWashed(false)
                    .notFilterRecord()
                    .copyBizTags()
                    .build();

            HttpRequest httpRequest = crawlerItemRequestRecord.getHttpRequest();
            httpRequest.setMethod("POST");
            Map<String,Object> params = new HashMap<>();
            params.put("spuId",spuId);

            String sign = genSignature(params);
            params.put("sign",sign);
            HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
            httpRequest.setRequestBody(httpRequestBody);

            httpRequest.addHeader("Host","app.dewu.com");
            httpRequest.addHeader("Connection","keep-alive");
            httpRequest.addHeader("User-Agent",genRandomUA());
            httpRequest.addHeader("Content-Type","application/json");
            httpRequest.addHeader("platform","h5");
            httpRequest.addHeader("AppId","wxapp");
            httpRequest.addHeader("appVersion","4.4.0");
            httpRequest.addHeader("charset","utf-8");
            httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
            httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
            httpRequest.addHeader("wxapp-login-token","");

            parsedLinks.add(crawlerItemRequestRecord);
        }catch (Exception e){
            logger.error(e.getMessage());
            crawlerRequestRecord.setNeedWashPage(false);
            parsedLinks.add(crawlerRequestRecord);
        }

        return parsedLinks;
    }

    private List<CrawlerRequestRecord> parseMonthLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        JSONObject pageObj = JSONObject.parseObject(httpPage.getRawText());
        JSONObject dataObj = pageObj.getJSONObject("data");
        JSONArray allSells = dataObj.getJSONArray("list");
        for (Object allSell : allSells) {
            JSONObject allSellObj = (JSONObject)allSell;
            String sellDate = allSellObj.getString("date");
            JSONArray sells = allSellObj.getJSONArray("sells");
            for (Object sell : sells) {
                JSONObject sellObj = (JSONObject)sell;
                String productId = sellObj.getString("productId");
                String sourceName = sellObj.getString("sourceName");
                if (Integer.parseInt(productId) == 0){
                    genSellDetails(crawlerRequestRecord,sellObj,sellDate,parsedLinks);
                    continue;
                }
                String url = productDetailUrl + "#productId="+productId;
                CrawlerRequestRecord productRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .httpUrl(url)
                        .httpHead("appid","wxapp")
                        .httpHead("content-type","application/json")
                        .recordKey(productDetailUrl+productId)
                        .releaseTime(System.currentTimeMillis())
                        .notFilterRecord()
                        .needWashed(false)
                        .copyBizTags()
                        .build();
                HttpRequest httpRequest = productRecord.getHttpRequest();
                httpRequest.setMethod("POST");
                Map<String,Object> params = new HashMap<>();
                params.put("spuId",productId);
                params.put("productSourceName",sourceName);
                params.put("propertyValueId","0");

                String sign = genSignature(params);
                params.put("sign",sign);
                HttpRequestBody httpRequestBody = HttpRequestBody.json(JSON.toJSONString(params), "utf-8");
                httpRequest.setRequestBody(httpRequestBody);
                productRecord.tagsCreator().bizTags().addCustomKV("sell_object",sellObj);
                productRecord.tagsCreator().bizTags().addCustomKV(SELL_DATE,sellDate);

                httpRequest.addHeader("Host","app.dewu.com");
                httpRequest.addHeader("Connection","keep-alive");
                httpRequest.addHeader("User-Agent",genRandomUA());
                httpRequest.addHeader("Content-Type","application/json");
                httpRequest.addHeader("platform","h5");
                httpRequest.addHeader("AppId","wxapp");
                httpRequest.addHeader("appVersion","4.4.0");
                httpRequest.addHeader("charset","utf-8");
                httpRequest.addHeader("Accept-Encoding","gzip,compress,br,deflate");
                httpRequest.addHeader("Referer","https://servicewechat.com/wx3c12cdd0ae8b1a7b/225/page-frame.html");
                httpRequest.addHeader("wxapp-login-token","");

                parsedLinks.add(productRecord);
            }
        }
        return parsedLinks;
    }

    private String genRandomUA(){
        int part1 = RandomUtils.nextInt(1000, 9999);
        int part2 = RandomUtils.nextInt(1000, 1999);
        int part3 = RandomUtils.nextInt(100, 999);

        return String.format("Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2797 MMWEBSDK/20210501 Mobile Safari/537.36 MMWEBID/%s MicroMessenger/8.0.6.%s(0x28000%s) Process/appbrand0 WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64 MiniProgramEnv/android",
                part1,part2,part3);
    }

    private void genSellDetails(CrawlerRequestRecord crawlerRequestRecord, JSONObject sellObj, String sellDate, List<CrawlerRequestRecord> parsedLinks) {
        String sellId = sellObj.getString("sellId");
        String sellDetailUrl = String.format(sellDetailUrlFormat, sellId, "");
        Map<String, Object> urlParams = getUrlParams(sellDetailUrl);
        if (null != urlParams){
            String sign = genSignature(urlParams,"048a9c4943398714b356a696503d2d36");
            sellDetailUrl = sellDetailUrl + sign;
            CrawlerRequestRecord sellDetailRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .httpUrl(sellDetailUrl)
                    .recordKey(sellDetailUrl)
                    .releaseTime(System.currentTimeMillis())
                    .notFilterRecord()
                    .copyBizTags()
//                    .needParsed(false)
                    .build();
            sellDetailRecord.tagsCreator().bizTags().addCustomKV(SELL_DATE,sellDate);
            sellDetailRecord.tagsCreator().bizTags().addCustomKV(SELL_ID,sellId);
            parsedLinks.add(sellDetailRecord);
        }
    }

    private Map<String, Object> genPostSignNeedParams(String productId, String sourceName, String timestamp) {
        Map<String, Object> params = new HashMap<>();
        Map<String, Object> t1 = new HashMap<>();
        Map<String, Object> t2 = new HashMap<>();
        t1.put("name","commodity_QA");
        t1.put("value","0");
        t2.put("name","evaluate_466_new");
        t2.put("value","0");
        List<Map<String,Object>> tests = new ArrayList<>();
        tests.add(t1);
        tests.add(t2);
        params.put("abTests",tests);
        params.put("arFileSwitch",true);
        params.put("groupFirstId",Integer.parseInt(productId));
        params.put("loginToken","");
        params.put("newSign","");
        params.put("platform","android");
        params.put("productSourceName",sourceName);
        params.put("propertyValueId",0);
        params.put("skuId",0);
        params.put("spuId",Integer.parseInt(productId));
        params.put("timestamp",timestamp);
        params.put("uuid","c3e92d44fe7c3f9e");
        params.put("v","4.68.6");
        return params;
    }

    private List<CrawlerRequestRecord> parseIndexLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        JSONObject pageObj = JSONObject.parseObject(httpPage.getRawText());
        JSONObject dataObj = pageObj.getJSONObject("data");
        JSONArray months = dataObj.getJSONArray("monthList");
        for (Object month : months) {
            JSONObject monthObj = (JSONObject)month;
            String sellDate = monthObj.getString("date");
            String searchMonthUrl = String.format(searchMonthUrlFormat, sellDate, "");
            Map<String, Object> urlParams = getUrlParams(searchMonthUrl);
            if (null != urlParams){
                long timeMillis = System.currentTimeMillis();
                Map<String, Object> params = genSignNeedParams(urlParams, String.valueOf(timeMillis));
                String newSign = genNewSignature(params, scKey);
                searchMonthUrl = searchMonthUrl + newSign;

                CrawlerRequestRecord searchMonthRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .httpUrl(searchMonthUrl)
                        .releaseTime(timeMillis)
                        .notFilterRecord()
                        .copyBizTags()
                        .build();
                HttpRequest httpRequest = searchMonthRecord.getHttpRequest();
                httpRequest.setHeaders(genRequestHeaders(String.valueOf(timeMillis)));
                parsedLinks.add(searchMonthRecord);
            }
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        HttpRequest httpRequest = crawlerRecord.getHttpRequest();
        String httpRequestUrl = httpRequest.getUrl();
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (!page.isDownloadSuccess() || page.getStatusCode() != 200){
            logger.error("download failed! status code is [{}]",page.getStatusCode());
            return crawlerDataList;
        }
        if (httpRequestUrl.matches(productDetailUrlRegex)){
            washProductDetail(crawlerRecord,page,crawlerDataList);
        }
        if (httpRequestUrl.matches(sellDetailUrlRegex)){
            washSellDetail(crawlerRecord,page,crawlerDataList);
        }
        if (httpRequestUrl.matches(askProductUrl)){
            washProductInfo(crawlerDataList,crawlerRecord,page);
        }
        return crawlerDataList;
    }

    private void washSellDetail(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerData> crawlerDataList) {
        String sellid = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(SELL_ID);

        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .dataId(StringUtils.joinWith("-", domain(),SITE, CrawlerEnum.CrawlerDataType.article.enumVal(), sellid))
                .url("https://m.dewu.com/hybird/h5baseService/SellDetail?sellId=" + sellid)
                .releaseTime(System.currentTimeMillis())
                .content(httpPage.getRawText())
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();
        crawlerData.tagsCreator().bizTags().addSite(SITE);
//        crawlerData.tagsCreator().bizTags().addSiteBiz(site_biz);
        crawlerDataList.add(crawlerData);
    }

    private void washProductDetail(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerData> crawlerDataList) {
        String spuId = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(PRODUCT_ID);

        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .dataId(StringUtils.joinWith("-", domain(),SITE, CrawlerEnum.CrawlerDataType.article.enumVal(), spuId))
                .url("http://m.poizon.com/router/product/ProductDetail?spuId="+spuId+"&sourceName=shareDetail")
                .releaseTime(System.currentTimeMillis())
                .content(httpPage.getRawText())
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();
        crawlerData.tagsCreator().bizTags().addSite(SITE);
//        crawlerData.tagsCreator().bizTags().addSiteBiz(site_biz);
        crawlerDataList.add(crawlerData);
    }

    private void washProductInfo(List<CrawlerData> crawlerDataList,CrawlerRequestRecord crawlerRecord, HttpPage page){

        String spuId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagProductId);
        String productDetail = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagProductDetail);
        String sellProduct = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(tagSellProduct);
        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRecord, page)
                .dataId(StringUtils.joinWith("-", domain(),SITE, CrawlerEnum.CrawlerDataType.article.enumVal(), spuId))
                .url("http://m.poizon.com/router/product/ProductDetail?spuId="+spuId+"&sourceName=shareDetail")
                .releaseTime(System.currentTimeMillis())
                .addContentKV(tagProductDetail,productDetail)
                .addContentKV(tagSellProduct, sellProduct)
                .addContentKV(tagAskProduct, page.getRawText())
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();
        crawlerDataList.add(crawlerData);
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(startUrlRegex);
        addUrlRegular(searchMonthUrlRegex);
        addUrlRegular(productDetailUrlRegex);
        addUrlRegular(sellDetailUrlRegex);
        addUrlRegular(sellProductUrl);
        addUrlRegular(askProductUrl);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();
        String crawlerSite = categoryTag.getKVTagStrVal("site");
        return crawlerSite.equalsIgnoreCase(SITE);
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public String domain() {
        return DOMAIN;
    }

    /**
     * 转换map到url参数字符串
     * @param source url参数的map
     * @return 拼接好的url参数字符串
     */
    public static String asUrlParams(Map<String, Object> source){
        Iterator<String> it = source.keySet().iterator();
        StringBuilder paramStr = new StringBuilder();
        while (it.hasNext()){
            String key = it.next();
            String value = (String) source.get(key);
            if (StringUtils.isBlank(value)){
                continue;
            }
            try {
                // URL 编码
                value = URLEncoder.encode(value, "utf-8");
            } catch (UnsupportedEncodingException e) {
                // do nothing
                logger.error("url encode error");
            }
            paramStr.append("&").append(key).append("=").append(value);
        }
        // 去掉第一个&
        return paramStr.substring(1);
    }

    /**
     * 将url参数转换成map
     * @param url http://*.*.com?aa=11&bb=22&cc=33
     * @return map
     */
    public static Map<String, Object> getUrlParams(String url) {
        Map<String, Object> map = new HashMap<>();
        String param = null;
        if (url.contains("?")){
            param = url.split("\\?")[1];
        }
        if (StringUtils.isBlank(param)) {
            return null;
        }
        String[] params = param.split("&");
        for (String s : params) {
            String[] p = s.split("=");
            if (p.length == 2) {
                map.put(p[0], p[1]);
            }else {
                map.put(p[0],"");
            }
        }
        return map;
    }

    public static String genNewSignature(Map<String,Object> params, String scKey) {
        if(params == null){
            return null;
        }
        String sign = "";
        StringBuffer sb = new StringBuffer();
        try {
            // 1. 字典升序排序
            SortedMap<String,Object> sortedMap = new TreeMap<>(params);
            // 2. 拼按URL键值对
            Set<String> keySet = sortedMap.keySet();
            for(String key : keySet){
                //sign不参与算法
                if(key.equals("newSign")){
                    continue;
                }
                Object obj =  sortedMap.get(key);
                String value = obj.toString() ;
                if (obj instanceof Map){
                    value = JSON.toJSONString(obj);
                }
                if (obj instanceof List){
                    List list = (List) obj;
                    value = "";
                    for (Object o : list) {
                        if (value.length() > 0){
                            value += ",";
                        }
                        value += JSON.toJSONString(o);
                    }
                }
                if ("[]".equals(value)){
                    value = "";
                }
                sb.append(key).append(value);
            }
            String uriString = sb.toString();
            //AES运算
            AESUtils util = new AESUtils(scKey); // 密钥
            String encrypt = util.encryptData(uriString);
            encrypt = encrypt.replace("\r","").replace("\n","");//AES加密后存在换行符 需要去除后进行md5加密
            // 3. MD5运算得到请求签名
            sign = md5(encrypt);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }

    public static String genSignature(Map<String,Object> params) {
        return genSignature(params,"19bc545a393a25177083d4a748807cc0");
    }

    public static String genSignature(Map<String,Object> params,String salt) {
        if(params == null){
            return null;
        }
        String sign = "";
        StringBuffer sb = new StringBuffer();
        try {
            // 1. 字典升序排序
            SortedMap<String,Object> sortedMap = new TreeMap<>(params);
            // 2. 拼按URL键值对
            Set<String> keySet = sortedMap.keySet();
            for(String key : keySet){
                //sign不参与算法
                if(key.equals("sign")){
                    continue;
                }

                Object obj =  sortedMap.get(key);
                String value = obj.toString() ;
                if (obj instanceof Map){
                    value = JSON.toJSONString(obj);
                }
                if (obj instanceof List){
                    List list = (List) obj;
                    value = "";
                    for (Object o : list) {
                        if (value.length() > 0){
                            value += ",";
                        }
                        value += JSON.toJSONString(o);
                    }
//                    value = ","+JSON.toJSONString(obj);
                }
//                if ("[]".equals(value)){
//                    value = "";
//                }
                sb.append(key).append(value);

            }
            String uriString = sb.toString() + salt;
            // 3. MD5运算得到请求签名
            sign = md5(uriString);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }

    private Map<String, String> genRequestHeaders(String timestamp) {
        Map<String,String> headers = new HashMap<>();
        headers.put("duuuid","c3e92d44fe7c3f9e");
        headers.put("duplatform","android");
        headers.put("duv","4.68.6");
        headers.put("duloginToken","");
        headers.put("timestamp",timestamp);
        headers.put("Connection","Keep-Alive");
        headers.put("Accept-Encoding","gzip");
        return headers;
    }

    private Map<String, Object> genSignNeedParams(Map<String, Object> urlParams,String timestamp) {
        Map<String, Object> params = new HashMap<>();
        Set<Map.Entry<String, Object>> entries = urlParams.entrySet();
        for (Map.Entry<String, Object> entry : entries) {
            params.put(entry.getKey(),entry.getValue());
        }

        params.put("uuid","c3e92d44fe7c3f9e");
        params.put("platform","android");
        params.put("v","4.68.6");
        params.put("loginToken","");
        params.put("timestamp",timestamp);//header中的时间戳
        return params;
    }

    /**
     * 检查页面下载是否成功、完整
     * @param crawlerRequestRecord last record
     * @param httpPage page
     * @return boolean for page check
     */
    private boolean doHttpPageCheck(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        HttpRequest lastRequest = crawlerRequestRecord.getHttpRequest();
        String lastRequestUrl = lastRequest.getUrl();
        int statusCode = httpPage.getStatusCode();
        if (statusCode != 200){
            logger.error("download page {} error, status code is {}",lastRequestUrl,statusCode);
            return true;
        }
        if (!httpPage.isDownloadSuccess()){
            logger.error("download page failed, check your link {}",lastRequestUrl);
            return true;
        }
        if (StringUtils.isBlank(httpPage.getRawText())){
            logger.error("download page empty, check your link {}",lastRequestUrl);
            return true;
        }
        return false;
    }
}

class AESUtils {
    /**
     * 密钥算法
     */
    private static final String ALGORITHM = "AES";
    /**
     * 加解密算法/工作模式/填充方式
     */
    private static final String ALGORITHM_STR = "AES/ECB/PKCS5Padding";

    /**
     * SecretKeySpec类是KeySpec接口的实现类,用于构建秘密密钥规范
     */
    private SecretKeySpec key;

    public AESUtils(String hexKey) {
        key = new SecretKeySpec(hexKey.getBytes(), ALGORITHM);
    }

    /**
     * AES加密
     * @param data
     * @return
     * @throws Exception
     */
    public String encryptData(String data) throws Exception {
        Cipher cipher = Cipher.getInstance(ALGORITHM_STR); // 创建密码器
        cipher.init(Cipher.ENCRYPT_MODE, key);// 初始化
        return new BASE64Encoder().encode(cipher.doFinal(data.getBytes()));
    }

    /**
     * AES解密
     * @param base64Data
     * @return
     * @throws Exception
     */
    public String decryptData(String base64Data) throws Exception{
        Cipher cipher = Cipher.getInstance(ALGORITHM_STR);
        cipher.init(Cipher.DECRYPT_MODE, key);
        return new String(cipher.doFinal(new BASE64Decoder().decodeBuffer(base64Data)));
    }

    /**
     * hex字符串 转 byte数组
     * @param s
     * @return
     */
    private static byte[] hex2byte(String s) {
        if (s.length() % 2 == 0) {
            return hex2byte (s.getBytes(), 0, s.length() >> 1);
        } else {
            return hex2byte("0"+s);
        }
    }

    private static byte[] hex2byte (byte[] b, int offset, int len) {
        byte[] d = new byte[len];
        for (int i=0; i<len*2; i++) {
            int shift = i%2 == 1 ? 0 : 4;
            d[i>>1] |= Character.digit((char) b[offset+i], 16) << shift;
        }
        return d;
    }
}
